# -*- coding: utf-8 -*-
"""
Created on Wed Dec  6 22:09:03 2017

@author: Luther
"""

import os
import requests
from datetime import datetime
from tkinter import *
from tkinter import scrolledtext
from tkinter.filedialog import asksaveasfilename


class App(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.grid()
        self.MODE = IntVar()
        self.MODE.set(0)
        self.MODE_DICT = {0: "false", 1: "true"}
        self.createWidgets()

    def createWidgets(self):
        self.kwLabel = Label(self, text="请输入关键词：")
        self.kwLabel.grid(row=0, sticky=E)
        self.kwEntry = Entry(self, width=50)
        self.kwEntry.grid(row=0, column=1, columnspan=2, sticky=W)
        self.sdLabel = Label(self, text="请输入开始日期(YYYY-MM-DD)：")
        self.sdLabel.grid(row=1, sticky=E)
        self.sdEntry = Entry(self, width=50)
        self.sdEntry.grid(row=1, column=1, columnspan=2, sticky=W)
        self.edLabel = Label(self, text="请输入结束日期(YYYY-MM-DD)：")
        self.edLabel.grid(row=2, sticky=E)
        self.edEntry = Entry(self, width=50)
        self.edEntry.grid(row=2, column=1, columnspan=2, sticky=W)
        self.typLabel = Label(self, text="请输入是否要全文搜索：")
        self.typLabel.grid(row=3, sticky=E)
        self.typSelectorTrue = Radiobutton(
            self, text="是", value=1, variable=self.MODE)
        self.typSelectorTrue.grid(row=3, column=1)
        self.typSelectorFalse = Radiobutton(
            self, text="否", value=0, variable=self.MODE)
        self.typSelectorFalse.grid(row=3, column=2)
        self.fileLabel = Label(self, text="请选择输出文件位置：")
        self.fileLabel.grid(row=4, sticky=E)
        self.fileSelector = Entry(self, width=50)
        self.fileSelector.grid(row=4, column=1, columnspan=2, sticky=W)
        self.browseButton = Button(self, text="浏览", command=self.browse)
        self.browseButton.grid(row=4, column=3)
        self.startButton = Button(self, text="开始检索", command=self.click)
        self.startButton.grid(row=5, columnspan=4)
        self.consoleText = scrolledtext.ScrolledText(self)
        self.consoleText.grid(row=6, columnspan=4)
        self.consoleText.config(state=DISABLED)

    def print(self, text):
        self.consoleText.config(state=NORMAL)
        self.consoleText.insert(END, text + "\n")
        self.consoleText.config(state=DISABLED)
        self.consoleText.see(END)
        self.update()
    
    def text2log(self, path):
        text = self.consoleText.get("0.0", END)
        with open(path, 'w') as f:
            f.write(text)

    def browse(self):
        Filelist = asksaveasfilename(defaultextension='.txt', filetypes=[("CSV File",".csv")])
        self.fileSelector.delete(0, 'end')
        self.fileSelector.insert(0, Filelist)

    def click(self):
        keyword = self.kwEntry.get()
        sdate = self.sdEntry.get()
        edate = self.edEntry.get()
        outputpath = self.fileSelector.get()
        isfulltext = self.MODE_DICT[self.MODE.get()]
        self.search(keyword, sdate, edate, outputpath, isfulltext)

    def search(self, keyword, sdate, edate, outputpath, isfulltext):

        self.print("当前关键词：{}".format(keyword))
        self.print("起始日期：{} ~ {}".format(sdate, edate))
        mode_dict = {'true':'全文搜索','false':"标题搜索"}
        self.print("搜索模式：{}".format(mode_dict[isfulltext]))

        with open(outputpath, 'w') as f:
            if self.MODE.get()==1:
                f.write("code,name,time,title,content,pdf\n")
            else:
                f.write("code,name,time,title,pdf\n")

        i = 1
        url = "http://www.cninfo.com.cn/cninfo-new/fulltextSearch/full?searchkey={0:}&sdate={1:}&edate={2:}&isfulltext={4:}&sortName=nothing&sortType=desc&pageNum={3:}".format(
            keyword, sdate, edate, i, isfulltext)
        try:
            r = requests.get(url, timeout=30)
            r.raise_for_status
            pages = r.json()['totalpages']
            totalrec = r.json()['totalAnnouncement']
            self.print("获取总记录数成功！共{}页{}条记录".format(pages, totalrec))
        except:
            self.print("获取总页数失败，请重试！")

        for x in range(pages):
            i = 1 + x
            url = "http://www.cninfo.com.cn/cninfo-new/fulltextSearch/full?searchkey={0:}&sdate={1:}&edate={2:}&isfulltext={4:}&sortName=nothing&sortType=desc&pageNum={3:}".format(
                keyword, sdate, edate, i, isfulltext)
            try:
                r = requests.get(url, timeout=30)
                r.raise_for_status
                l = r.json()['announcements']
                for n in range(len(l)):
                    try:
                        d = l[n]
                        code = d['secCode']
                        name = d['secName']
                        title = d['announcementTitle'].replace(
                            "<em>", "").replace("</em>", "").replace(",", "")
                        time = datetime.fromtimestamp(
                            d['announcementTime'] / 1000).strftime('%Y-%m-%d')
                        pdf = "http://www.cninfo.com.cn/" + d['adjunctUrl']
                        if isfulltext == "true":
                            content = d['announcementContent'].replace(
                                "<em>", "").replace("</em>", "").replace(
                                    ",", "")
                            ret = code + "," + name + "," + time + "," + title + "," + content + "," + pdf + "\n"
                        else:
                            ret = code + "," + name + "," + time + "," + title + "," + pdf + "\n"
                        #                    print(ret)
                        with open(outputpath, 'a') as f:
                            f.write(ret)
                    except:
                        pass
                self.print("第{:}页爬取成功！".format(i))

            except:
                self.print("第{:}页爬取失败！".format(i))
        self.print("完成！")
        self.print("结果保存至：{}".format(outputpath))
        logpath = os.path.join(os.path.dirname(outputpath),"log.log")
        self.print("日志文件位于：{}".format(logpath))
        self.text2log(logpath)


if __name__ == "__main__":
    app = App()
    app.master.title("巨潮公告爬取程序")
    app.mainloop()
